{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "bc3c372a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "id": "345a6fab",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train=pd.read_csv('train.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "4b153faf",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test=pd.read_csv('test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "3df52aee",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(404290, 6) (2345796, 3)\n"
     ]
    }
   ],
   "source": [
    "print(df_train.shape,df_test.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "043d8c4c",
   "metadata": {},
   "source": [
    "观察是否存在缺失值情况。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "993a7aa4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "question1    1\n",
       "question2    2\n",
       "dtype: int64"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.isnull().sum()[df_train.isnull().sum()>0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "id": "5ad756b3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>qid1</th>\n",
       "      <th>qid2</th>\n",
       "      <th>question1</th>\n",
       "      <th>question2</th>\n",
       "      <th>is_duplicate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>363362</th>\n",
       "      <td>363362</td>\n",
       "      <td>493340</td>\n",
       "      <td>493341</td>\n",
       "      <td>NaN</td>\n",
       "      <td>My Chinese name is Haichao Yu. What English na...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            id    qid1    qid2 question1  \\\n",
       "363362  363362  493340  493341       NaN   \n",
       "\n",
       "                                                question2  is_duplicate  \n",
       "363362  My Chinese name is Haichao Yu. What English na...             0  "
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train[df_train['question1'].isnull()==True]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "id": "ee1a18ae",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>qid1</th>\n",
       "      <th>qid2</th>\n",
       "      <th>question1</th>\n",
       "      <th>question2</th>\n",
       "      <th>is_duplicate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>105780</th>\n",
       "      <td>105780</td>\n",
       "      <td>174363</td>\n",
       "      <td>174364</td>\n",
       "      <td>How can I develop android app?</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201841</th>\n",
       "      <td>201841</td>\n",
       "      <td>303951</td>\n",
       "      <td>174364</td>\n",
       "      <td>How can I create an Android app?</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            id    qid1    qid2                         question1 question2  \\\n",
       "105780  105780  174363  174364    How can I develop android app?       NaN   \n",
       "201841  201841  303951  174364  How can I create an Android app?       NaN   \n",
       "\n",
       "        is_duplicate  \n",
       "105780             0  \n",
       "201841             0  "
      ]
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train[df_train['question2'].isnull()==True]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e6e8ba9e",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "8ce9939c",
   "metadata": {},
   "source": [
    "question1和question2都有缺失值，有缺失值应该认为是不一样的问题，这里删除这些行。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "df603b13",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train=df_train[df_train['question1'].isnull()==False]\n",
    "df_train=df_train[df_train['question2'].isnull()==False]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "e7ccf70a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(404287, 6)"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "850b2e84",
   "metadata": {},
   "source": [
    "**meta-features**\n",
    "* number of common tokens(non-stopwords) in both the questions\n",
    "* tokens count difference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "ac89df4f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'!\"#$%&\\'()*+,-./:;<=>?@[\\\\]^_`{|}~'"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "string.punctuation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "c6ec5149",
   "metadata": {},
   "outputs": [],
   "source": [
    "import re, string, six\n",
    "\n",
    "from nltk.corpus import stopwords\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "re_tok = re.compile(f'([{string.punctuation}“”¨«»®´·º½¾¿¡§£₤‘’])')# 以 f开头表示在字符串内支持大括号内的python 表达式\n",
    "\n",
    "def tokenize(s): \n",
    "    return re_tok.sub(r' \\1 ', s).split()#去掉反斜杠的转移机制\n",
    "\n",
    "def clean_text(s):\n",
    "    try:\n",
    "        return re.sub(r'[^A-Za-z0-9,?\"\\'. ]+', '', s).encode('utf-8').decode('utf-8').lower()\n",
    "    except:\n",
    "        return \"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "2301629f",
   "metadata": {},
   "outputs": [],
   "source": [
    "stops = set(stopwords.words(\"english\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "d0d9cfa5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "127\n",
      "{'am', 'did', 'being', 'him', 'just', 'again', 'has', 'once', 'each', 'not', 'our', 'at', 'how', 'over', 'is', 'up', 'few', 'an', 'such', 'no', 'there', 'me', 'doing', 'than', 'about', 'but', 'it', 'some', 'them', 'above', 'of', 'with', 'after', 'whom', 'from', 'were', 'as', 'to', 'because', 'having', 'all', 'for', 'these', 'its', 'where', 'own', 'was', 'most', 'very', 'your', 'off', 'a', 'too', 'will', 'then', 'their', 'here', 'why', 'both', 'only', 'itself', 'yourself', 'myself', 'under', 'have', 'and', 'through', 'should', 'or', 'be', 'had', 'themselves', 'ours', 'while', 'before', 'you', 'so', 'he', 'by', 'nor', 'which', 'don', 'more', 'her', 'on', 'if', 'in', 'ourselves', 'that', 'she', 's', 'hers', 'into', 'what', 'down', 'during', 'further', 'herself', 'this', 't', 'his', 'theirs', 'we', 'now', 'out', 'below', 'been', 'same', 'himself', 'my', 'between', 'who', 'when', 'other', 'yours', 'against', 'they', 'does', 'until', 'any', 'are', 'i', 'do', 'can', 'the', 'yourselves', 'those'}\n"
     ]
    }
   ],
   "source": [
    "print(len(stops))\n",
    "print(stops)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "3f801b9a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def word_match_share(row):\n",
    "    q1words = {}\n",
    "    q2words = {}\n",
    "    try:\n",
    "        for word in tokenize(row['question1']):\n",
    "            if word not in stops:\n",
    "                q1words[word] = 1\n",
    "        for word in tokenize(row['question2']):\n",
    "            if word not in stops:\n",
    "                q2words[word] = 1\n",
    "        if len(q1words) == 0 or len(q2words) == 0:\n",
    "            return 0\n",
    "        shared_words_in_q1 = [w for w in q1words.keys() if w in q2words]\n",
    "        shared_words_in_q2 = [w for w in q2words.keys() if w in q1words]\n",
    "        return (len(shared_words_in_q1) + len(shared_words_in_q2))/(len(q1words) + len(q2words))\n",
    "    except:\n",
    "        return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "e96b9597",
   "metadata": {},
   "outputs": [],
   "source": [
    "def word_count_diff(row):\n",
    "    try:\n",
    "        q1words = len(list(filter(lambda x: x.lower() not in stops, tokenize(row['question1']))))\n",
    "        q2words = len(list(filter(lambda x: x.lower() not in stops, tokenize(row['question2']))))\n",
    "        return abs(q1words - q2words)\n",
    "    except:\n",
    "        return 50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "27067190",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train['wms'] = df_train.apply(word_match_share, axis=1)\n",
    "df_train['wcd'] = df_train.apply(word_count_diff, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "dccdf57e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>qid1</th>\n",
       "      <th>qid2</th>\n",
       "      <th>question1</th>\n",
       "      <th>question2</th>\n",
       "      <th>is_duplicate</th>\n",
       "      <th>wms</th>\n",
       "      <th>wcd</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>What is the step by step guide to invest in sh...</td>\n",
       "      <td>What is the step by step guide to invest in sh...</td>\n",
       "      <td>0</td>\n",
       "      <td>0.933333</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>What is the story of Kohinoor (Koh-i-Noor) Dia...</td>\n",
       "      <td>What would happen if the Indian government sto...</td>\n",
       "      <td>0</td>\n",
       "      <td>0.640000</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>How can I increase the speed of my internet co...</td>\n",
       "      <td>How can Internet speed be increased by hacking...</td>\n",
       "      <td>0</td>\n",
       "      <td>0.375000</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>Why am I mentally very lonely? How can I solve...</td>\n",
       "      <td>Find the remainder when [math]23^{24}[/math] i...</td>\n",
       "      <td>0</td>\n",
       "      <td>0.095238</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>Which one dissolve in water quikly sugar, salt...</td>\n",
       "      <td>Which fish would survive in salt water?</td>\n",
       "      <td>0</td>\n",
       "      <td>0.400000</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  qid1  qid2                                          question1  \\\n",
       "0   0     1     2  What is the step by step guide to invest in sh...   \n",
       "1   1     3     4  What is the story of Kohinoor (Koh-i-Noor) Dia...   \n",
       "2   2     5     6  How can I increase the speed of my internet co...   \n",
       "3   3     7     8  Why am I mentally very lonely? How can I solve...   \n",
       "4   4     9    10  Which one dissolve in water quikly sugar, salt...   \n",
       "\n",
       "                                           question2  is_duplicate       wms  \\\n",
       "0  What is the step by step guide to invest in sh...             0  0.933333   \n",
       "1  What would happen if the Indian government sto...             0  0.640000   \n",
       "2  How can Internet speed be increased by hacking...             0  0.375000   \n",
       "3  Find the remainder when [math]23^{24}[/math] i...             0  0.095238   \n",
       "4            Which fish would survive in salt water?             0  0.400000   \n",
       "\n",
       "   wcd  \n",
       "0    1  \n",
       "1    5  \n",
       "2    1  \n",
       "3   14  \n",
       "4    7  "
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "38ef314a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>is_duplicate</th>\n",
       "      <th>wcd</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>3.058971</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1.514843</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   is_duplicate       wcd\n",
       "0             0  3.058971\n",
       "1             1  1.514843"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.groupby(['is_duplicate']).agg({'wcd': np.mean}).reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "bfeb4640",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train['question1'] = df_train['question1'].apply(lambda x: clean_text(x))\n",
    "df_train['question2'] = df_train['question2'].apply(lambda x: clean_text(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "f433a57d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>qid1</th>\n",
       "      <th>qid2</th>\n",
       "      <th>question1</th>\n",
       "      <th>question2</th>\n",
       "      <th>is_duplicate</th>\n",
       "      <th>wms</th>\n",
       "      <th>wcd</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>what is the step by step guide to invest in sh...</td>\n",
       "      <td>what is the step by step guide to invest in sh...</td>\n",
       "      <td>0</td>\n",
       "      <td>0.933333</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>what is the story of kohinoor kohinoor diamond?</td>\n",
       "      <td>what would happen if the indian government sto...</td>\n",
       "      <td>0</td>\n",
       "      <td>0.640000</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>how can i increase the speed of my internet co...</td>\n",
       "      <td>how can internet speed be increased by hacking...</td>\n",
       "      <td>0</td>\n",
       "      <td>0.375000</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>why am i mentally very lonely? how can i solve...</td>\n",
       "      <td>find the remainder when math2324math is divide...</td>\n",
       "      <td>0</td>\n",
       "      <td>0.095238</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>which one dissolve in water quikly sugar, salt...</td>\n",
       "      <td>which fish would survive in salt water?</td>\n",
       "      <td>0</td>\n",
       "      <td>0.400000</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>404285</th>\n",
       "      <td>404285</td>\n",
       "      <td>433578</td>\n",
       "      <td>379845</td>\n",
       "      <td>how many keywords are there in the racket prog...</td>\n",
       "      <td>how many keywords are there in perl programmin...</td>\n",
       "      <td>0</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>404286</th>\n",
       "      <td>404286</td>\n",
       "      <td>18840</td>\n",
       "      <td>155606</td>\n",
       "      <td>do you believe there is life after death?</td>\n",
       "      <td>is it true that there is life after death?</td>\n",
       "      <td>1</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>404287</th>\n",
       "      <td>404287</td>\n",
       "      <td>537928</td>\n",
       "      <td>537929</td>\n",
       "      <td>what is one coin?</td>\n",
       "      <td>what's this coin?</td>\n",
       "      <td>0</td>\n",
       "      <td>0.750000</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>404288</th>\n",
       "      <td>404288</td>\n",
       "      <td>537930</td>\n",
       "      <td>537931</td>\n",
       "      <td>what is the approx annual cost of living while...</td>\n",
       "      <td>i am having little hairfall problem but i want...</td>\n",
       "      <td>0</td>\n",
       "      <td>0.133333</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>404289</th>\n",
       "      <td>404289</td>\n",
       "      <td>537932</td>\n",
       "      <td>537933</td>\n",
       "      <td>what is like to have sex with cousin?</td>\n",
       "      <td>what is it like to have sex with your cousin?</td>\n",
       "      <td>0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>404287 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            id    qid1    qid2  \\\n",
       "0            0       1       2   \n",
       "1            1       3       4   \n",
       "2            2       5       6   \n",
       "3            3       7       8   \n",
       "4            4       9      10   \n",
       "...        ...     ...     ...   \n",
       "404285  404285  433578  379845   \n",
       "404286  404286   18840  155606   \n",
       "404287  404287  537928  537929   \n",
       "404288  404288  537930  537931   \n",
       "404289  404289  537932  537933   \n",
       "\n",
       "                                                question1  \\\n",
       "0       what is the step by step guide to invest in sh...   \n",
       "1         what is the story of kohinoor kohinoor diamond?   \n",
       "2       how can i increase the speed of my internet co...   \n",
       "3       why am i mentally very lonely? how can i solve...   \n",
       "4       which one dissolve in water quikly sugar, salt...   \n",
       "...                                                   ...   \n",
       "404285  how many keywords are there in the racket prog...   \n",
       "404286          do you believe there is life after death?   \n",
       "404287                                  what is one coin?   \n",
       "404288  what is the approx annual cost of living while...   \n",
       "404289              what is like to have sex with cousin?   \n",
       "\n",
       "                                                question2  is_duplicate  \\\n",
       "0       what is the step by step guide to invest in sh...             0   \n",
       "1       what would happen if the indian government sto...             0   \n",
       "2       how can internet speed be increased by hacking...             0   \n",
       "3       find the remainder when math2324math is divide...             0   \n",
       "4                 which fish would survive in salt water?             0   \n",
       "...                                                   ...           ...   \n",
       "404285  how many keywords are there in perl programmin...             0   \n",
       "404286         is it true that there is life after death?             1   \n",
       "404287                                  what's this coin?             0   \n",
       "404288  i am having little hairfall problem but i want...             0   \n",
       "404289      what is it like to have sex with your cousin?             0   \n",
       "\n",
       "             wms  wcd  \n",
       "0       0.933333    1  \n",
       "1       0.640000    5  \n",
       "2       0.375000    1  \n",
       "3       0.095238   14  \n",
       "4       0.400000    7  \n",
       "...          ...  ...  \n",
       "404285  0.666667    0  \n",
       "404286  0.600000    0  \n",
       "404287  0.750000    0  \n",
       "404288  0.133333    5  \n",
       "404289  1.000000    0  \n",
       "\n",
       "[404287 rows x 8 columns]"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "11cc8fca",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "seed = 12345\n",
    "\n",
    "X = df_train.iloc[:,6:]\n",
    "Y = df_train['is_duplicate']\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, Y, test_size=0.20, random_state=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "4489d6bc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>wms</th>\n",
       "      <th>wcd</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.933333</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.640000</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.375000</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.095238</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.400000</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>404285</th>\n",
       "      <td>0.666667</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>404286</th>\n",
       "      <td>0.600000</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>404287</th>\n",
       "      <td>0.750000</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>404288</th>\n",
       "      <td>0.133333</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>404289</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>404287 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             wms  wcd\n",
       "0       0.933333    1\n",
       "1       0.640000    5\n",
       "2       0.375000    1\n",
       "3       0.095238   14\n",
       "4       0.400000    7\n",
       "...          ...  ...\n",
       "404285  0.666667    0\n",
       "404286  0.600000    0\n",
       "404287  0.750000    0\n",
       "404288  0.133333    5\n",
       "404289  1.000000    0\n",
       "\n",
       "[404287 rows x 2 columns]"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "id": "0b546cda",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.metrics import mean_absolute_error\n",
    "from sklearn.metrics import log_loss\n",
    "import lightgbm as lgb\n",
    "\n",
    "# create dataset for lightgbm\n",
    "lgb_train = lgb.Dataset(X_train, y_train)\n",
    "lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "e4a72bd5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] Auto-choosing row-wise multi-threading, the overhead of testing was 0.017061 seconds.\n",
      "You can set `force_row_wise=true` to remove the overhead.\n",
      "And if memory is not enough, you can set `force_col_wise=true`.\n",
      "[1]\tvalid_0's binary_logloss: 0.647628\n",
      "Training until validation scores don't improve for 5 rounds\n",
      "[2]\tvalid_0's binary_logloss: 0.638773\n",
      "[3]\tvalid_0's binary_logloss: 0.630717\n",
      "[4]\tvalid_0's binary_logloss: 0.623371\n",
      "[5]\tvalid_0's binary_logloss: 0.616635\n",
      "[6]\tvalid_0's binary_logloss: 0.610486\n",
      "[7]\tvalid_0's binary_logloss: 0.604852\n",
      "[8]\tvalid_0's binary_logloss: 0.599674\n",
      "[9]\tvalid_0's binary_logloss: 0.594904\n",
      "[10]\tvalid_0's binary_logloss: 0.590519\n",
      "[11]\tvalid_0's binary_logloss: 0.586492\n",
      "[12]\tvalid_0's binary_logloss: 0.582782\n",
      "[13]\tvalid_0's binary_logloss: 0.579345\n",
      "[14]\tvalid_0's binary_logloss: 0.576196\n",
      "[15]\tvalid_0's binary_logloss: 0.573282\n",
      "[16]\tvalid_0's binary_logloss: 0.570592\n",
      "[17]\tvalid_0's binary_logloss: 0.568101\n",
      "[18]\tvalid_0's binary_logloss: 0.565804\n",
      "[19]\tvalid_0's binary_logloss: 0.563667\n",
      "[20]\tvalid_0's binary_logloss: 0.561672\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[20]\tvalid_0's binary_logloss: 0.561672\n"
     ]
    }
   ],
   "source": [
    "# specify your configurations as a dict\n",
    "params = {'boosting_type': 'gbdt','objective': 'binary','metric': 'binary_logloss',\n",
    "          'num_leaves': 31,'learning_rate': 0.05,'feature_fraction': 0.9,\n",
    "          'bagging_fraction': 0.8,'bagging_freq': 5,'verbose': 0}\n",
    "gbm = lgb.train(params,lgb_train,num_boost_round=20,valid_sets=lgb_eval,\n",
    "                early_stopping_rounds=5)\n",
    "# predict\n",
    "y_pred = gbm.predict(X_test, num_iteration=gbm.best_iteration)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "a97fd754",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5616717339249052"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "log_loss(y_test,y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "76be1cbb",
   "metadata": {},
   "outputs": [],
   "source": [
    "from bayes_opt import BayesianOptimization\n",
    "from sklearn.model_selection import cross_val_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "id": "e79897ea",
   "metadata": {},
   "outputs": [],
   "source": [
    "def lgb_cv(n_estimators,min_split_gain,subsample, max_depth,colsample_bytree, min_child_samples,reg_alpha,reg_lambda,num_leaves,learning_rate):\n",
    "        model = lgb.LGBMClassifier(boosting_type='gbdt', objective='binary',n_jobs=-1,\n",
    "                                   colsample_bytree=float(colsample_bytree),\n",
    "                                   min_child_samples=int(min_child_samples),\n",
    "                                   n_estimators=int(n_estimators),\n",
    "                                   num_leaves=int(num_leaves),\n",
    "                                   reg_alpha=float(reg_alpha),\n",
    "                                   reg_lambda=float(reg_lambda),\n",
    "                                   max_depth=int(max_depth),\n",
    "                                   subsample=float(subsample),\n",
    "                                   min_split_gain = float(min_split_gain),\n",
    "                                   learning_rate=float(learning_rate),\n",
    "                                   )\n",
    "        cv_score = cross_val_score(model, X_train, y_train, scoring='neg_log_loss', cv=5).mean()\n",
    "        return cv_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "id": "f10894a3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 实例化BayesianOptimization类，参数靠自己去定义取值范围\n",
    "lgb_bo = BayesianOptimization(\n",
    "        lgb_cv,\n",
    "        {\n",
    "        'colsample_bytree': (0.5,1),\n",
    "        'min_child_samples': (2, 200),\n",
    "        'num_leaves': (5, 1000),\n",
    "        'subsample': (0.6, 1),\n",
    "        'max_depth':(2,10),\n",
    "        'n_estimators': (10, 1000),\n",
    "        'reg_alpha':(0,10),\n",
    "        'reg_lambda':(0,10),\n",
    "        'min_split_gain':(0,1),\n",
    "        'learning_rate':(0,1)\n",
    "         },\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "id": "76c92606",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "|   iter    |  target   | colsam... | learni... | max_depth | min_ch... | min_sp... | n_esti... | num_le... | reg_alpha | reg_la... | subsample |\n",
      "-------------------------------------------------------------------------------------------------------------------------------------------------\n",
      "| \u001b[0m 1       \u001b[0m | \u001b[0m-0.5404  \u001b[0m | \u001b[0m 0.5129  \u001b[0m | \u001b[0m 0.05168 \u001b[0m | \u001b[0m 2.257   \u001b[0m | \u001b[0m 125.1   \u001b[0m | \u001b[0m 0.484   \u001b[0m | \u001b[0m 798.9   \u001b[0m | \u001b[0m 946.9   \u001b[0m | \u001b[0m 6.353   \u001b[0m | \u001b[0m 9.638   \u001b[0m | \u001b[0m 0.624   \u001b[0m |\n",
      "| \u001b[95m 2       \u001b[0m | \u001b[95m-0.5354  \u001b[0m | \u001b[95m 0.8488  \u001b[0m | \u001b[95m 0.3836  \u001b[0m | \u001b[95m 5.855   \u001b[0m | \u001b[95m 121.0   \u001b[0m | \u001b[95m 0.5177  \u001b[0m | \u001b[95m 125.6   \u001b[0m | \u001b[95m 6.014   \u001b[0m | \u001b[95m 0.3221  \u001b[0m | \u001b[95m 5.561   \u001b[0m | \u001b[95m 0.8049  \u001b[0m |\n",
      "| \u001b[95m 3       \u001b[0m | \u001b[95m-0.5349  \u001b[0m | \u001b[95m 0.8857  \u001b[0m | \u001b[95m 0.9014  \u001b[0m | \u001b[95m 7.641   \u001b[0m | \u001b[95m 115.2   \u001b[0m | \u001b[95m 0.5408  \u001b[0m | \u001b[95m 646.2   \u001b[0m | \u001b[95m 477.2   \u001b[0m | \u001b[95m 4.582   \u001b[0m | \u001b[95m 5.43    \u001b[0m | \u001b[95m 0.6737  \u001b[0m |\n",
      "| \u001b[95m 4       \u001b[0m | \u001b[95m-0.5346  \u001b[0m | \u001b[95m 0.9325  \u001b[0m | \u001b[95m 0.04133 \u001b[0m | \u001b[95m 9.189   \u001b[0m | \u001b[95m 128.4   \u001b[0m | \u001b[95m 0.6931  \u001b[0m | \u001b[95m 885.7   \u001b[0m | \u001b[95m 352.3   \u001b[0m | \u001b[95m 3.022   \u001b[0m | \u001b[95m 1.162   \u001b[0m | \u001b[95m 0.8601  \u001b[0m |\n",
      "| \u001b[0m 5       \u001b[0m | \u001b[0m-0.5353  \u001b[0m | \u001b[0m 0.8599  \u001b[0m | \u001b[0m 0.6532  \u001b[0m | \u001b[0m 4.823   \u001b[0m | \u001b[0m 6.859   \u001b[0m | \u001b[0m 0.5175  \u001b[0m | \u001b[0m 351.9   \u001b[0m | \u001b[0m 324.2   \u001b[0m | \u001b[0m 4.073   \u001b[0m | \u001b[0m 5.198   \u001b[0m | \u001b[0m 0.6475  \u001b[0m |\n",
      "| \u001b[0m 6       \u001b[0m | \u001b[0m-0.5398  \u001b[0m | \u001b[0m 0.6812  \u001b[0m | \u001b[0m 0.8469  \u001b[0m | \u001b[0m 6.474   \u001b[0m | \u001b[0m 118.6   \u001b[0m | \u001b[0m 0.7749  \u001b[0m | \u001b[0m 131.9   \u001b[0m | \u001b[0m 12.01   \u001b[0m | \u001b[0m 2.214   \u001b[0m | \u001b[0m 1.216   \u001b[0m | \u001b[0m 0.7231  \u001b[0m |\n",
      "| \u001b[0m 7       \u001b[0m | \u001b[0m-0.5393  \u001b[0m | \u001b[0m 0.6066  \u001b[0m | \u001b[0m 0.7297  \u001b[0m | \u001b[0m 6.681   \u001b[0m | \u001b[0m 39.93   \u001b[0m | \u001b[0m 0.02043 \u001b[0m | \u001b[0m 821.6   \u001b[0m | \u001b[0m 82.9    \u001b[0m | \u001b[0m 3.601   \u001b[0m | \u001b[0m 3.284   \u001b[0m | \u001b[0m 0.8331  \u001b[0m |\n",
      "| \u001b[0m 8       \u001b[0m | \u001b[0m-0.5394  \u001b[0m | \u001b[0m 0.626   \u001b[0m | \u001b[0m 0.7365  \u001b[0m | \u001b[0m 4.828   \u001b[0m | \u001b[0m 39.24   \u001b[0m | \u001b[0m 0.1259  \u001b[0m | \u001b[0m 205.3   \u001b[0m | \u001b[0m 973.5   \u001b[0m | \u001b[0m 0.2989  \u001b[0m | \u001b[0m 7.616   \u001b[0m | \u001b[0m 0.7559  \u001b[0m |\n",
      "| \u001b[0m 9       \u001b[0m | \u001b[0m-0.5397  \u001b[0m | \u001b[0m 0.7483  \u001b[0m | \u001b[0m 0.517   \u001b[0m | \u001b[0m 2.633   \u001b[0m | \u001b[0m 173.5   \u001b[0m | \u001b[0m 0.1086  \u001b[0m | \u001b[0m 875.1   \u001b[0m | \u001b[0m 666.2   \u001b[0m | \u001b[0m 6.597   \u001b[0m | \u001b[0m 1.589   \u001b[0m | \u001b[0m 0.9269  \u001b[0m |\n",
      "| \u001b[0m 10      \u001b[0m | \u001b[0m-0.5395  \u001b[0m | \u001b[0m 0.5792  \u001b[0m | \u001b[0m 0.4751  \u001b[0m | \u001b[0m 6.498   \u001b[0m | \u001b[0m 159.1   \u001b[0m | \u001b[0m 0.02323 \u001b[0m | \u001b[0m 66.91   \u001b[0m | \u001b[0m 170.3   \u001b[0m | \u001b[0m 4.557   \u001b[0m | \u001b[0m 7.798   \u001b[0m | \u001b[0m 0.9655  \u001b[0m |\n",
      "| \u001b[0m 11      \u001b[0m | \u001b[0m-0.5352  \u001b[0m | \u001b[0m 0.8314  \u001b[0m | \u001b[0m 0.8635  \u001b[0m | \u001b[0m 3.43    \u001b[0m | \u001b[0m 73.89   \u001b[0m | \u001b[0m 0.05723 \u001b[0m | \u001b[0m 320.7   \u001b[0m | \u001b[0m 653.9   \u001b[0m | \u001b[0m 9.299   \u001b[0m | \u001b[0m 3.51    \u001b[0m | \u001b[0m 0.9162  \u001b[0m |\n",
      "| \u001b[0m 12      \u001b[0m | \u001b[0m-0.5396  \u001b[0m | \u001b[0m 0.5735  \u001b[0m | \u001b[0m 0.6551  \u001b[0m | \u001b[0m 7.321   \u001b[0m | \u001b[0m 114.9   \u001b[0m | \u001b[0m 0.3038  \u001b[0m | \u001b[0m 628.6   \u001b[0m | \u001b[0m 908.7   \u001b[0m | \u001b[0m 3.773   \u001b[0m | \u001b[0m 8.609   \u001b[0m | \u001b[0m 0.6465  \u001b[0m |\n",
      "| \u001b[95m 13      \u001b[0m | \u001b[95m-0.5344  \u001b[0m | \u001b[95m 0.8618  \u001b[0m | \u001b[95m 0.8888  \u001b[0m | \u001b[95m 8.302   \u001b[0m | \u001b[95m 124.0   \u001b[0m | \u001b[95m 0.1434  \u001b[0m | \u001b[95m 218.0   \u001b[0m | \u001b[95m 37.37   \u001b[0m | \u001b[95m 1.38    \u001b[0m | \u001b[95m 8.755   \u001b[0m | \u001b[95m 0.7302  \u001b[0m |\n",
      "| \u001b[0m 14      \u001b[0m | \u001b[0m-0.5347  \u001b[0m | \u001b[0m 0.8808  \u001b[0m | \u001b[0m 0.4924  \u001b[0m | \u001b[0m 4.016   \u001b[0m | \u001b[0m 20.14   \u001b[0m | \u001b[0m 0.07866 \u001b[0m | \u001b[0m 777.0   \u001b[0m | \u001b[0m 659.1   \u001b[0m | \u001b[0m 3.726   \u001b[0m | \u001b[0m 3.798   \u001b[0m | \u001b[0m 0.7984  \u001b[0m |\n",
      "| \u001b[0m 15      \u001b[0m | \u001b[0m-0.5401  \u001b[0m | \u001b[0m 0.7078  \u001b[0m | \u001b[0m 0.5977  \u001b[0m | \u001b[0m 4.747   \u001b[0m | \u001b[0m 162.2   \u001b[0m | \u001b[0m 0.7066  \u001b[0m | \u001b[0m 957.3   \u001b[0m | \u001b[0m 544.1   \u001b[0m | \u001b[0m 6.978   \u001b[0m | \u001b[0m 8.736   \u001b[0m | \u001b[0m 0.9745  \u001b[0m |\n",
      "| \u001b[0m 16      \u001b[0m | \u001b[0m-0.5353  \u001b[0m | \u001b[0m 0.9299  \u001b[0m | \u001b[0m 0.432   \u001b[0m | \u001b[0m 3.648   \u001b[0m | \u001b[0m 31.71   \u001b[0m | \u001b[0m 0.1297  \u001b[0m | \u001b[0m 187.2   \u001b[0m | \u001b[0m 508.9   \u001b[0m | \u001b[0m 7.538   \u001b[0m | \u001b[0m 6.972   \u001b[0m | \u001b[0m 0.9217  \u001b[0m |\n",
      "| \u001b[0m 17      \u001b[0m | \u001b[0m-0.5349  \u001b[0m | \u001b[0m 0.8467  \u001b[0m | \u001b[0m 0.8758  \u001b[0m | \u001b[0m 9.995   \u001b[0m | \u001b[0m 135.6   \u001b[0m | \u001b[0m 0.8666  \u001b[0m | \u001b[0m 312.1   \u001b[0m | \u001b[0m 100.4   \u001b[0m | \u001b[0m 3.164   \u001b[0m | \u001b[0m 2.402   \u001b[0m | \u001b[0m 0.9103  \u001b[0m |\n",
      "| \u001b[0m 18      \u001b[0m | \u001b[0m-0.5351  \u001b[0m | \u001b[0m 0.769   \u001b[0m | \u001b[0m 0.7042  \u001b[0m | \u001b[0m 7.466   \u001b[0m | \u001b[0m 177.2   \u001b[0m | \u001b[0m 0.6202  \u001b[0m | \u001b[0m 498.0   \u001b[0m | \u001b[0m 579.4   \u001b[0m | \u001b[0m 1.481   \u001b[0m | \u001b[0m 7.215   \u001b[0m | \u001b[0m 0.8989  \u001b[0m |\n",
      "| \u001b[0m 19      \u001b[0m | \u001b[0m-0.5397  \u001b[0m | \u001b[0m 0.6716  \u001b[0m | \u001b[0m 0.9571  \u001b[0m | \u001b[0m 4.81    \u001b[0m | \u001b[0m 60.72   \u001b[0m | \u001b[0m 0.2939  \u001b[0m | \u001b[0m 600.6   \u001b[0m | \u001b[0m 56.77   \u001b[0m | \u001b[0m 4.689   \u001b[0m | \u001b[0m 9.977   \u001b[0m | \u001b[0m 0.6949  \u001b[0m |\n",
      "| \u001b[0m 20      \u001b[0m | \u001b[0m-0.5399  \u001b[0m | \u001b[0m 0.6389  \u001b[0m | \u001b[0m 0.3946  \u001b[0m | \u001b[0m 7.543   \u001b[0m | \u001b[0m 34.39   \u001b[0m | \u001b[0m 0.5706  \u001b[0m | \u001b[0m 719.6   \u001b[0m | \u001b[0m 856.6   \u001b[0m | \u001b[0m 9.512   \u001b[0m | \u001b[0m 3.496   \u001b[0m | \u001b[0m 0.7056  \u001b[0m |\n",
      "| \u001b[0m 21      \u001b[0m | \u001b[0m-0.5403  \u001b[0m | \u001b[0m 0.721   \u001b[0m | \u001b[0m 0.05086 \u001b[0m | \u001b[0m 5.67    \u001b[0m | \u001b[0m 23.08   \u001b[0m | \u001b[0m 0.8679  \u001b[0m | \u001b[0m 600.5   \u001b[0m | \u001b[0m 774.6   \u001b[0m | \u001b[0m 9.851   \u001b[0m | \u001b[0m 0.5802  \u001b[0m | \u001b[0m 0.681   \u001b[0m |\n",
      "| \u001b[0m 22      \u001b[0m | \u001b[0m-0.5355  \u001b[0m | \u001b[0m 0.9156  \u001b[0m | \u001b[0m 0.3655  \u001b[0m | \u001b[0m 4.836   \u001b[0m | \u001b[0m 124.7   \u001b[0m | \u001b[0m 0.4939  \u001b[0m | \u001b[0m 883.8   \u001b[0m | \u001b[0m 361.8   \u001b[0m | \u001b[0m 5.414   \u001b[0m | \u001b[0m 2.117   \u001b[0m | \u001b[0m 0.6424  \u001b[0m |\n",
      "| \u001b[0m 23      \u001b[0m | \u001b[0m-0.5399  \u001b[0m | \u001b[0m 0.6834  \u001b[0m | \u001b[0m 0.1106  \u001b[0m | \u001b[0m 6.029   \u001b[0m | \u001b[0m 68.59   \u001b[0m | \u001b[0m 0.4547  \u001b[0m | \u001b[0m 374.2   \u001b[0m | \u001b[0m 613.7   \u001b[0m | \u001b[0m 6.405   \u001b[0m | \u001b[0m 6.025   \u001b[0m | \u001b[0m 0.8418  \u001b[0m |\n",
      "| \u001b[0m 24      \u001b[0m | \u001b[0m-0.5395  \u001b[0m | \u001b[0m 0.5444  \u001b[0m | \u001b[0m 0.3956  \u001b[0m | \u001b[0m 8.305   \u001b[0m | \u001b[0m 159.3   \u001b[0m | \u001b[0m 0.3779  \u001b[0m | \u001b[0m 861.7   \u001b[0m | \u001b[0m 574.4   \u001b[0m | \u001b[0m 2.407   \u001b[0m | \u001b[0m 1.069   \u001b[0m | \u001b[0m 0.6967  \u001b[0m |\n",
      "| \u001b[0m 25      \u001b[0m | \u001b[0m-0.5396  \u001b[0m | \u001b[0m 0.7362  \u001b[0m | \u001b[0m 0.6238  \u001b[0m | \u001b[0m 3.476   \u001b[0m | \u001b[0m 81.79   \u001b[0m | \u001b[0m 0.1182  \u001b[0m | \u001b[0m 741.0   \u001b[0m | \u001b[0m 527.6   \u001b[0m | \u001b[0m 3.692   \u001b[0m | \u001b[0m 4.865   \u001b[0m | \u001b[0m 0.8672  \u001b[0m |\n",
      "| \u001b[0m 26      \u001b[0m | \u001b[0m-0.5396  \u001b[0m | \u001b[0m 0.5063  \u001b[0m | \u001b[0m 0.3223  \u001b[0m | \u001b[0m 7.171   \u001b[0m | \u001b[0m 134.4   \u001b[0m | \u001b[0m 0.5825  \u001b[0m | \u001b[0m 943.9   \u001b[0m | \u001b[0m 361.8   \u001b[0m | \u001b[0m 1.432   \u001b[0m | \u001b[0m 3.854   \u001b[0m | \u001b[0m 0.7514  \u001b[0m |\n",
      "| \u001b[0m 27      \u001b[0m | \u001b[0m-0.5347  \u001b[0m | \u001b[0m 0.9524  \u001b[0m | \u001b[0m 0.668   \u001b[0m | \u001b[0m 8.765   \u001b[0m | \u001b[0m 6.275   \u001b[0m | \u001b[0m 0.2738  \u001b[0m | \u001b[0m 720.3   \u001b[0m | \u001b[0m 983.8   \u001b[0m | \u001b[0m 4.937   \u001b[0m | \u001b[0m 3.689   \u001b[0m | \u001b[0m 0.6432  \u001b[0m |\n",
      "| \u001b[0m 28      \u001b[0m | \u001b[0m-0.5398  \u001b[0m | \u001b[0m 0.727   \u001b[0m | \u001b[0m 0.643   \u001b[0m | \u001b[0m 5.512   \u001b[0m | \u001b[0m 198.7   \u001b[0m | \u001b[0m 0.3377  \u001b[0m | \u001b[0m 641.8   \u001b[0m | \u001b[0m 714.6   \u001b[0m | \u001b[0m 4.566   \u001b[0m | \u001b[0m 9.161   \u001b[0m | \u001b[0m 0.8178  \u001b[0m |\n",
      "| \u001b[0m 29      \u001b[0m | \u001b[0m-0.5355  \u001b[0m | \u001b[0m 0.8196  \u001b[0m | \u001b[0m 0.6403  \u001b[0m | \u001b[0m 6.453   \u001b[0m | \u001b[0m 64.63   \u001b[0m | \u001b[0m 0.9758  \u001b[0m | \u001b[0m 862.1   \u001b[0m | \u001b[0m 593.3   \u001b[0m | \u001b[0m 6.863   \u001b[0m | \u001b[0m 1.682   \u001b[0m | \u001b[0m 0.6207  \u001b[0m |\n",
      "| \u001b[0m 30      \u001b[0m | \u001b[0m-0.5348  \u001b[0m | \u001b[0m 0.873   \u001b[0m | \u001b[0m 0.8371  \u001b[0m | \u001b[0m 6.98    \u001b[0m | \u001b[0m 25.6    \u001b[0m | \u001b[0m 0.3749  \u001b[0m | \u001b[0m 926.2   \u001b[0m | \u001b[0m 770.8   \u001b[0m | \u001b[0m 4.183   \u001b[0m | \u001b[0m 3.367   \u001b[0m | \u001b[0m 0.6184  \u001b[0m |\n",
      "=================================================================================================================================================\n"
     ]
    }
   ],
   "source": [
    "# 训练\n",
    "lgb_bo.maximize()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "id": "b868a13a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'target': -0.5343728330020127, 'params': {'colsample_bytree': 0.8617558650288266, 'learning_rate': 0.8887569455441605, 'max_depth': 8.301685196278695, 'min_child_samples': 124.02249604271428, 'min_split_gain': 0.14340188311502522, 'n_estimators': 218.00430917531367, 'num_leaves': 37.367526479056494, 'reg_alpha': 1.3798749213580985, 'reg_lambda': 8.75479513746093, 'subsample': 0.7301959165813758}}\n"
     ]
    }
   ],
   "source": [
    "print(lgb_bo.max)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "id": "af748cab",
   "metadata": {},
   "outputs": [],
   "source": [
    "lgb_train = lgb.Dataset(X_train, y_train)\n",
    "lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "id": "29ea220b",
   "metadata": {},
   "outputs": [],
   "source": [
    "params={'boosting_type': 'gbdt','objective': 'binary','metric': 'binary_logloss',\n",
    "        'colsample_bytree': lgb_bo.max['params']['colsample_bytree'], \n",
    "        'learning_rate': lgb_bo.max['params']['learning_rate'], \n",
    "        'max_depth': int(round(lgb_bo.max['params']['max_depth'])), \n",
    "        'min_child_samples': int(round(lgb_bo.max['params']['min_child_samples'])), \n",
    "        'min_split_gain': lgb_bo.max['params']['min_split_gain'],\n",
    "        'n_estimators': int(round(lgb_bo.max['params']['n_estimators'])), \n",
    "        'num_leaves': int(round(lgb_bo.max['params']['num_leaves'])), \n",
    "        'reg_alpha': lgb_bo.max['params']['reg_alpha'], \n",
    "        'reg_lambda': lgb_bo.max['params']['reg_lambda'], \n",
    "        'subsample': lgb_bo.max['params']['subsample']}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "id": "c109d290",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'boosting_type': 'gbdt',\n",
       " 'objective': 'binary',\n",
       " 'metric': 'binary_logloss',\n",
       " 'colsample_bytree': 0.8617558650288266,\n",
       " 'learning_rate': 0.8887569455441605,\n",
       " 'max_depth': 8,\n",
       " 'min_child_samples': 124,\n",
       " 'min_split_gain': 0.14340188311502522,\n",
       " 'n_estimators': 218,\n",
       " 'num_leaves': 37,\n",
       " 'reg_alpha': 1.3798749213580985,\n",
       " 'reg_lambda': 8.75479513746093,\n",
       " 'subsample': 0.7301959165813758}"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "id": "c1dfbee7",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Administrator\\anaconda3\\lib\\site-packages\\lightgbm\\engine.py:148: UserWarning: Found `n_estimators` in params. Will use it instead of argument\n",
      "  _log_warning(\"Found `{}` in params. Will use it instead of argument\".format(alias))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Info] Number of positive: 119584, number of negative: 203845\n",
      "[LightGBM] [Warning] Auto-choosing col-wise multi-threading, the overhead of testing was 0.106863 seconds.\n",
      "You can set `force_col_wise=true` to remove the overhead.\n",
      "[LightGBM] [Info] Total Bins 238\n",
      "[LightGBM] [Info] Number of data points in the train set: 323429, number of used features: 2\n",
      "[LightGBM] [Info] [binary:BoostFromScore]: pavg=0.369738 -> initscore=-0.533341\n",
      "[LightGBM] [Info] Start training from score -0.533341\n",
      "[1]\tvalid_0's binary_logloss: 0.553709\n",
      "Training until validation scores don't improve for 5 rounds\n",
      "[2]\tvalid_0's binary_logloss: 0.540305\n",
      "[3]\tvalid_0's binary_logloss: 0.536778\n",
      "[4]\tvalid_0's binary_logloss: 0.535903\n",
      "[5]\tvalid_0's binary_logloss: 0.535457\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[6]\tvalid_0's binary_logloss: 0.535284\n",
      "[7]\tvalid_0's binary_logloss: 0.535165\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[8]\tvalid_0's binary_logloss: 0.534941\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[9]\tvalid_0's binary_logloss: 0.534787\n",
      "[10]\tvalid_0's binary_logloss: 0.534801\n",
      "[11]\tvalid_0's binary_logloss: 0.534735\n",
      "[12]\tvalid_0's binary_logloss: 0.534676\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[13]\tvalid_0's binary_logloss: 0.534671\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] Stopped training because there are no more leaves that meet the split requirements\n",
      "[14]\tvalid_0's binary_logloss: 0.534671\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] Stopped training because there are no more leaves that meet the split requirements\n",
      "[15]\tvalid_0's binary_logloss: 0.534671\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] Stopped training because there are no more leaves that meet the split requirements\n",
      "[16]\tvalid_0's binary_logloss: 0.534671\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] Stopped training because there are no more leaves that meet the split requirements\n",
      "[17]\tvalid_0's binary_logloss: 0.534671\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] Stopped training because there are no more leaves that meet the split requirements\n",
      "[18]\tvalid_0's binary_logloss: 0.534671\n",
      "Early stopping, best iteration is:\n",
      "[13]\tvalid_0's binary_logloss: 0.534671\n"
     ]
    }
   ],
   "source": [
    "gbm = lgb.train(params,\n",
    "                lgb_train,\n",
    "                num_boost_round=20,\n",
    "                valid_sets=lgb_eval,\n",
    "                early_stopping_rounds=5)\n",
    "y_pred = gbm.predict(X_test, num_iteration=gbm.best_iteration)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1373b700",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
